import re

from bs4 import BeautifulSoup
from get_api_class import get_api_in_homepage
from spiders import spider


def get_class_details(response):
    modules = []
    web = "https://developer.android.google.cn"
    types = 'modules'
    resp = response.text
    soup = BeautifulSoup(resp, "html.parser")
    table = soup.find('table', class_='nested-classes responsive')
    h2 = table.find('h2').text
    # print("h2 :" + h2)
    tr_list = table.find_all('tr')
    # print(tr_list)
    for tr in tr_list:
        td = tr.find('td')
        try:
            a = td.find('a')
            url = web + a.get('href')
            flag = a.text
            if h2.find('Classes') != -1:
                types = 'classes'
            classify = (flag, url, types)
            modules.append(classify)
        except:
            continue
    return modules

def get_game_in_homepage(url_list):
    game_dict = dict()
    for overview, url in url_list:
        tasks = list()
        modules_list = list()
        if url =='invalid url':
            pass
        else:
            flag = [overview]
            task = (url, flag)
            tasks.append(task)
            ret = spider(20, tasks)
            for flag, response in ret:
                if response.status_code == 200:
                    # print(response.text)
                    try:
                        modules_list = get_class_details(response)
                    except:
                        modules_list = [(overview, url, 'modules')]
            if modules_list:
                game_dict[overview] = modules_list
    print(game_dict)
    return game_dict

def get_target_from_web(url):
    target_list = []
    web = "https://developer.android.google.cn"
    tasks = list()
    reference_list = []
    flag = ['target-url']
    task = (url, flag)
    tasks.append(task)
    ret = spider(20, tasks)
    for flag, response in ret:
        if response.status_code == 200:
            resp = response.text
            # print(resp)
            pattern_h2 = re.compile(r'<h2 (.*?)</h2>.*?<p>(.*?)</p>', re.M | re.S)
            pattern_h3 = re.compile(r'<h3 (.*?)</h3>.*?<p>(.*?)</p>', re.M | re.S)
            h2_content = pattern_h2.findall(resp)
            h3_content = pattern_h3.findall(resp)
            content = h2_content + h3_content
            # print(content)
            for x, y in content:
                # print(x)
                # print(y)
                pattern_name = re.compile(r'id="(.*)" data-text=.*')
                name = pattern_name.search(x).group(1)
                # print(name)
                pattern_url = re.compile(r'<a href="(\S+)"><strong>Reference</strong></a>')
                try:
                    url = web + pattern_url.search(y).group(1)
                except:
                    url = 'invalid url'
                # print(url)
                classify = (name, url)
                reference_list.append(classify)
    # print(reference_list)
    return reference_list



if __name__ == "__main__":
    url_list = get_target_from_web('https://developer.android.google.cn/games/agdk/libraries-overview')
    dict1 = get_api_in_homepage()
    dict2 = get_game_in_homepage(url_list)
    dict1.update(dict2)
    print(dict1)
    # language_list = [('C', 'https://gitee.com/explore/all?lang=C&order=starred'),
    #                  ('C++', 'https://gitee.com/explore/all?lang=cpp&order=starred')]
    # get_page_in_homepage(language_list)